
from collections import defaultdict
import json
from functools import partial
import numpy as np
from sklearn.cluster import DBSCAN
from sent2vec import Sent2VecEmbeddings
with open('/home/lxy/wxbdata/procedure.jsonl','r') as f:
    data=list(map(partial(json.loads),f.readlines()))

id_list=[d['id'] for d in data]

document=[d['name'] for d in data]

model=Sent2VecEmbeddings()
doc,maxNorm=model.embed_documents(document)
doc/=maxNorm
doc_embedding=np.array(doc)

print(doc_embedding,doc_embedding.shape)
k = 5  # 替换为你选择的簇数
dbscan = DBSCAN(eps=0.5, min_samples=1)
dbscan.fit(doc_embedding)
cluster_labels = dbscan.labels_  # 每个数据点的簇标签
print(cluster_labels,len(cluster_labels))
cluster_dict=defaultdict(list)
for i in range(len(cluster_labels)):
    cluster_dict[cluster_labels[i]].append([id_list[i],document[i]])

cluster_list=list(cluster_dict.values())
with open('cluster_output.txt','w',encoding='utf-8') as o:
    
    for i in range(len(cluster_list)):
        for j in range(len(cluster_list[i])):
            if j>=1:
                o.write("|")
            for k in range(len(cluster_list[i][j])):
                # 写入数组的值到文件，可以根据需要调整格式
                
                o.write(str(cluster_list[i][j][k]) + " ")
             
        # 写入空行，将不同的三维数组分隔开
        o.write("\n")

